#!/usr/bin/env python
# -*- coding:utf-8 -*-
import re
import sys
import time

import pymongo
import requests
from scpy.logger import get_logger
from scpy.xawesome_time import now, parse_time

from util import get_mongo_conf

reload(sys)
sys.setdefaultencoding("utf-8")


COUNT = 50

logger = get_logger(__file__)

PATTERN = re.compile(ur'<tr.+?<td.+?>(.+?)</td><td.+?>(\d{4}-\d{2}-\d{2})</td><td.+?>(.+?)</td><td.+?>(.+?)</td><td.+?>(.+?)</td><td.+?>(.*?)</td><td.+?>(.*?)</td>.*?</tr>', re.I)

MONGO_CONF = get_mongo_conf()
MONGO_CONN = pymongo.MongoClient(MONGO_CONF[0], MONGO_CONF[1])

request = requests.Session()
request.headers['User-Agent'] = 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/42.0.2311.90 Safari/537.36'


def get(url):
    time.sleep(0.5)
    return request.get(url).content


def find_a_page(page):
    url = 'http://www.ccopyright.com.cn/cpcc/ZRegisterAction.do?method=list&no=fck&sql_opusName=&sql_regnum=&sql_obligee=&sortLabel=registerDate&sortOrder=&curPage=%d&count=%s' % (page, COUNT)
    items = []
    content = unicode(re.sub(ur'\s', '', get(url), flags=re.I).decode('gb2312', 'ignore').encode('utf-8'))
    content = content[content.find(ur'nowrap>首次发表日期</th'):]
    for index, item in enumerate(PATTERN.findall(content)):
        try:
            items.append({
                'regNumber': item[0],
                'regDate': parse_time(item[1]),
                'entityName': item[2],
                'entityType': item[3],
                'owner': item[4],
                'finishDate': parse_time(item[5]),
                'publishDate': parse_time(item[6]),
            })
        except Exception, e:
            logger.exception(e)
    return items


def save(items):
    saved = 0
    coll = MONGO_CONN['crawler_company_all']['copyright']
    for item in items:
        item['_id'] = item['regNumber']
        item['updateTime'] = now()
        rst = coll.find_one_and_update(
            filter={'_id': item['_id']},
            update={'$set': item},
            upsert=True
        )
        if rst:
            saved += 1
    return saved


def main(start_page=1):
    current_page = start_page
    saved = 0
    while True:
        if saved > 30:
            break
        try:
            logger.info('find a new page %d' % current_page)
            items = find_a_page(current_page)
            if not items:
                logger.info('find nothing of page [%d]' % current_page)
                break
            logger.info('data count of page [%d] : [%s]' % (current_page, len(items)))
            saved += save(items)
        except KeyboardInterrupt, e:
            break
        except Exception, e:
            logger.exception(e)
        current_page += 1


if __name__ == '__main__':
    main()

